//////////////////////////////////////////////////////////////////
// (c) Copyright 2008- by Jeongnim Kim
//////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////
//   National Center for Supercomputing Applications &
//   Materials Computation Center
//   University of Illinois, Urbana-Champaign
//   Urbana, IL 61801
//   e-mail: jnkim@ncsa.uiuc.edu
//
// Supported by 
//   National Center for Supercomputing Applications, UIUC
//   Materials Computation Center, UIUC
//////////////////////////////////////////////////////////////////
// -*- C++ -*-
#include <mpi/communicator.h>
#include <cstdlib>
#include <iostream>
#include <sstream>

namespace mpi {

  bool environment::mpi_world_invalid=true;
  int environment::mpi_world_copies=0;

  environment::environment(int argc, char **argv)
  {
     environment::initialize(argc,argv);
  }

#if defined(HAVE_MPI)
  void environment::initialize(int argc, char **argv)
  {
    if(mpi_world_invalid)
    {
      MPI_Init(&argc,&argv);
      mpi_world_invalid=false;
    }
    //need to modification with OpenSHMEM
#if defined(ENABLE_SHMEM) && defined (_CRAY)
    shmem_init();
#endif
  }

  void environment::finalize()
  {
    if(mpi_world_invalid) return;
#if defined(ENABLE_SHMEM) && defined (_CRAY)
    shmem_finalize();
#endif
    MPI_Finalize();
    mpi_world_invalid=true;
  }

  communicator::communicator():comm_ptr(0)
  {
    environment::initialize();
    //not needed for shared_ptr
    //comm_ptr.reset(new MPI_Comm(MPI_COMM_WORLD));
    environment::mpi_world_copies++;
    comm_ptr=MPI_COMM_WORLD;
    MPI_Comm_size(comm_ptr,&d_ncontexts);
    MPI_Comm_rank(comm_ptr,&d_mycontext);
    MPI_Comm_group(comm_ptr,&d_groupid);

    if(d_mycontext) 
      my_logstream=new std::ostringstream();
    else
      my_logstream=&(std::cout);

    requests.resize(4*d_ncontexts);
    status.resize(4*d_ncontexts);
  }

  communicator::communicator(int argc, char **argv): comm_ptr(0)
  {
    environment::initialize(argc,argv);
    //not needed for shared_ptr
    //comm_ptr.reset(new MPI_Comm(MPI_COMM_WORLD));
    environment::mpi_world_copies++;
    comm_ptr=MPI_COMM_WORLD;
    MPI_Comm_size(comm_ptr,&d_ncontexts);
    MPI_Comm_rank(comm_ptr,&d_mycontext);
    MPI_Comm_group(comm_ptr,&d_groupid);
    //if(dims[0]*dims[1]!=0) MPI_Dims_create(d_ncontexts,2,dims);
    //
    if(d_mycontext) 
      my_logstream=new std::ostringstream();
    else
      my_logstream=&(std::cout);

    requests.resize(4*d_ncontexts);
    status.resize(4*d_ncontexts);
  }

  communicator::communicator(const MPI_Comm& comm, int oldrank)
  {
    MPI_Comm_dup(comm,&comm_ptr);
    MPI_Comm_size(comm_ptr,&d_ncontexts);
    MPI_Comm_rank(comm_ptr,&d_mycontext);
    MPI_Comm_group(comm_ptr,&d_groupid);

    int grank;
    MPI_Comm_rank(MPI_COMM_WORLD,&grank);

    if(grank!=oldrank)
      std::cerr << " rank mismatching " << std::endl;

    requests.resize(4*d_ncontexts);
    status.resize(4*d_ncontexts);
    //std::ostringstream os;
    //os << " group " << d_groupid << " oldrank " << oldrank 
    //  << " newrank " << d_mycontext << " size " << d_ncontexts << "\n";
    //std::cout <<os.str();
    //std::cout.flush();

    my_logstream=new std::ostringstream();
  }

  communicator communicator::split(int color) const
  {
    return split(color,rank());
  }

  communicator communicator::split(int color, int key) const
  {
    //int c_ncontexts, c_mycontext, c_groupid;
    MPI_Comm row;
    MPI_Comm_split(comm_ptr,color,key,&row);
    //MPI_Comm_size(row,&c_ncontexts);
    //MPI_Comm_rank(row,&c_mycontext);
    //MPI_Comm_group(row,&c_groupid);

    return communicator(row,rank());
  }

  communicator::~communicator()
  {
    if(my_logstream && my_logstream != &(std::cout)) delete my_logstream;
    if(comm_ptr && comm_ptr != MPI_COMM_WORLD)
      MPI_Comm_free(&comm_ptr);
    else
      environment::finalize();
  }

  void communicator::abort(int errcode)
  {
    MPI_Abort(MPI_COMM_WORLD,errcode);
  }

  void communicator::barrier()
  {
    MPI_Barrier(comm_ptr);
  }
#else
  //empty implementations
  void environment::initialize(int argc, char **argv)
  {
  }

  void environment::finalize()
  {
  }

  communicator::communicator()
    :comm_ptr(0),d_ncontexts(1),d_mycontext(0)
     ,my_logstream(&(std::cout))
  {
  }

  ///constructor with arguments
  communicator::communicator(int argc, char **argv)
    :comm_ptr(0),d_ncontexts(1),d_mycontext(0)
     ,my_logstream(&(std::cout))
  {
  }

  /**destructor
   * Call proper finalization of Communication library
   */
  communicator::~communicator()
  {
  }

  void communicator::abort(int errcode)
  {
    abort(errcode);
  }
  void communicator::barrier()
  {
  }
#endif
}///namespace
/***************************************************************************
 * $RCSfile$   $Author: jnkim $
 * $Revision: 2809 $   $Date: 2008-07-14 10:52:34 -0500 (Mon, 14 Jul 2008) $
 * $Id: communicator.h 2809 2008-07-14 15:52:34Z jnkim $ 
 ***************************************************************************/
